{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Multilabel-Bert.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ShreyasHavaldar7/GCN_Project/blob/master/Multilabel_Bert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W5tWtp4Ut41x",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "5c3ede55-f03e-4a98-a5db-569fc697b1b2"
      },
      "source": [
        "!pip install fast-bert"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting fast-bert\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0d/87/1c55c8ecc82f73470f3872a275099d7dec45013871986765d4883aa46a80/fast_bert-1.4.4-py3-none-any.whl (70kB)\n",
            "\r\u001b[K     |████▊                           | 10kB 14.6MB/s eta 0:00:01\r\u001b[K     |█████████▍                      | 20kB 1.8MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 30kB 2.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 40kB 1.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 51kB 2.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 61kB 2.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 71kB 2.3MB/s \n",
            "\u001b[?25hCollecting pytorch-lamb\n",
            "  Downloading https://files.pythonhosted.org/packages/43/98/3bce14a319317a2856db722f2542d329baf42845fa53563d0d749c5a2d40/pytorch_lamb-1.0.0-py3-none-any.whl\n",
            "Collecting transformers>=2.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fd/f9/51824e40f0a23a49eab4fcaa45c1c797cbf9761adedd0b558dab7c958b34/transformers-2.1.1-py3-none-any.whl (311kB)\n",
            "\u001b[K     |████████████████████████████████| 317kB 8.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from fast-bert) (1.3.0+cu100)\n",
            "Requirement already satisfied: fastai in /usr/local/lib/python3.6/dist-packages (from fast-bert) (1.0.58)\n",
            "Collecting tensorboardX\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/5c/e918d9f190baab8d55bad52840d8091dd5114cc99f03eaa6d72d404503cc/tensorboardX-1.9-py2.py3-none-any.whl (190kB)\n",
            "\u001b[K     |████████████████████████████████| 194kB 44.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch-lamb->fast-bert) (4.28.1)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (from pytorch-lamb->fast-bert) (0.4.1+cu100)\n",
            "Collecting regex\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ff/60/d9782c56ceefa76033a00e1f84cd8c586c75e6e7fea2cd45ee8b46a386c5/regex-2019.08.19-cp36-cp36m-manylinux1_x86_64.whl (643kB)\n",
            "\u001b[K     |████████████████████████████████| 645kB 42.8MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1f/8e/ed5364a06a9ba720fddd9820155cc57300d28f5f43a6fd7b7e817177e642/sacremoses-0.0.35.tar.gz (859kB)\n",
            "\u001b[K     |████████████████████████████████| 860kB 21.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->fast-bert) (2.21.0)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->fast-bert) (1.9.253)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->fast-bert) (1.17.3)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/3d/efb655a670b98f62ec32d66954e1109f403db4d937c50d779a75b9763a29/sentencepiece-0.1.83-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 41.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (3.1.1)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (4.3.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (3.13)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (1.3.1)\n",
            "Requirement already satisfied: spacy>=2.0.18 in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (2.1.8)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (4.6.3)\n",
            "Requirement already satisfied: numexpr in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (2.7.0)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (0.7)\n",
            "Requirement already satisfied: bottleneck in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (1.2.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (19.2)\n",
            "Requirement already satisfied: nvidia-ml-py3 in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (7.352.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (0.25.2)\n",
            "Requirement already satisfied: fastprogress>=0.1.19 in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert) (0.1.21)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorboardX->fast-bert) (3.10.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from tensorboardX->fast-bert) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.0.0->fast-bert) (7.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.0.0->fast-bert) (0.14.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->fast-bert) (2019.9.11)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->fast-bert) (1.24.3)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->fast-bert) (2.8)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->fast-bert) (3.0.4)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->fast-bert) (0.9.4)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->fast-bert) (0.2.1)\n",
            "Requirement already satisfied: botocore<1.13.0,>=1.12.253 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->fast-bert) (1.12.253)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai->fast-bert) (1.1.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai->fast-bert) (2.4.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai->fast-bert) (0.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai->fast-bert) (2.6.1)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from Pillow->fastai->fast-bert) (0.46)\n",
            "Requirement already satisfied: thinc<7.1.0,>=7.0.8 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert) (7.0.8)\n",
            "Requirement already satisfied: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert) (0.9.6)\n",
            "Requirement already satisfied: srsly<1.1.0,>=0.0.6 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert) (0.1.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert) (1.0.2)\n",
            "Requirement already satisfied: preshed<2.1.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert) (2.0.1)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert) (0.2.2)\n",
            "Requirement already satisfied: blis<0.3.0,>=0.2.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert) (0.2.4)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert) (2.0.2)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->fastai->fast-bert) (2018.9)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->tensorboardX->fast-bert) (41.4.0)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.253->boto3->transformers>=2.0.0->fast-bert) (0.15.2)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.35-cp36-none-any.whl size=883999 sha256=a456c6266f25b0accc8dd25fe420ec6741ff5c66b8ac2a352aac435a0d7cb038\n",
            "  Stored in directory: /root/.cache/pip/wheels/63/2a/db/63e2909042c634ef551d0d9ac825b2b0b32dede4a6d87ddc94\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tensorboardX, pytorch-lamb, regex, sacremoses, sentencepiece, transformers, fast-bert\n",
            "Successfully installed fast-bert-1.4.4 pytorch-lamb-1.0.0 regex-2019.8.19 sacremoses-0.0.35 sentencepiece-0.1.83 tensorboardX-1.9 transformers-2.1.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nwB3rstFuCne",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "462a908d-e98c-4570-bc38-2dd1461627e1"
      },
      "source": [
        "pip install git+https://github.com/kaushaltrivedi/fast-bert.git"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/kaushaltrivedi/fast-bert.git\n",
            "  Cloning https://github.com/kaushaltrivedi/fast-bert.git to /tmp/pip-req-build-el00pxzk\n",
            "  Running command git clone -q https://github.com/kaushaltrivedi/fast-bert.git /tmp/pip-req-build-el00pxzk\n",
            "Requirement already satisfied (use --upgrade to upgrade): fast-bert==1.4.4 from git+https://github.com/kaushaltrivedi/fast-bert.git in /usr/local/lib/python3.6/dist-packages\n",
            "Requirement already satisfied: torch>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from fast-bert==1.4.4) (1.3.0+cu100)\n",
            "Requirement already satisfied: fastai in /usr/local/lib/python3.6/dist-packages (from fast-bert==1.4.4) (1.0.58)\n",
            "Requirement already satisfied: pytorch-lamb in /usr/local/lib/python3.6/dist-packages (from fast-bert==1.4.4) (1.0.0)\n",
            "Requirement already satisfied: tensorboardX in /usr/local/lib/python3.6/dist-packages (from fast-bert==1.4.4) (1.9)\n",
            "Requirement already satisfied: transformers>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from fast-bert==1.4.4) (2.1.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch>=1.2.0->fast-bert==1.4.4) (1.17.3)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (4.3.0)\n",
            "Requirement already satisfied: nvidia-ml-py3 in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (7.352.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (4.6.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (3.1.1)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (3.13)\n",
            "Requirement already satisfied: fastprogress>=0.1.19 in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (0.1.21)\n",
            "Requirement already satisfied: spacy>=2.0.18 in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (2.1.8)\n",
            "Requirement already satisfied: numexpr in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (2.7.0)\n",
            "Requirement already satisfied: bottleneck in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (1.2.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (19.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (0.25.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (1.3.1)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (0.7)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (2.21.0)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (from fastai->fast-bert==1.4.4) (0.4.1+cu100)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch-lamb->fast-bert==1.4.4) (4.28.1)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorboardX->fast-bert==1.4.4) (3.10.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from tensorboardX->fast-bert==1.4.4) (1.12.0)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->fast-bert==1.4.4) (0.0.35)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->fast-bert==1.4.4) (0.1.83)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->fast-bert==1.4.4) (1.9.253)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->fast-bert==1.4.4) (2019.8.19)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from Pillow->fastai->fast-bert==1.4.4) (0.46)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai->fast-bert==1.4.4) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai->fast-bert==1.4.4) (1.1.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai->fast-bert==1.4.4) (2.6.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai->fast-bert==1.4.4) (2.4.2)\n",
            "Requirement already satisfied: srsly<1.1.0,>=0.0.6 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert==1.4.4) (0.1.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert==1.4.4) (2.0.2)\n",
            "Requirement already satisfied: thinc<7.1.0,>=7.0.8 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert==1.4.4) (7.0.8)\n",
            "Requirement already satisfied: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert==1.4.4) (0.9.6)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert==1.4.4) (1.0.2)\n",
            "Requirement already satisfied: preshed<2.1.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert==1.4.4) (2.0.1)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert==1.4.4) (0.2.2)\n",
            "Requirement already satisfied: blis<0.3.0,>=0.2.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.0.18->fastai->fast-bert==1.4.4) (0.2.4)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->fastai->fast-bert==1.4.4) (2018.9)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->fastai->fast-bert==1.4.4) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->fastai->fast-bert==1.4.4) (1.24.3)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->fastai->fast-bert==1.4.4) (2.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->fastai->fast-bert==1.4.4) (2019.9.11)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->tensorboardX->fast-bert==1.4.4) (41.4.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.0.0->fast-bert==1.4.4) (7.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.0.0->fast-bert==1.4.4) (0.14.0)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->fast-bert==1.4.4) (0.9.4)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->fast-bert==1.4.4) (0.2.1)\n",
            "Requirement already satisfied: botocore<1.13.0,>=1.12.253 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->fast-bert==1.4.4) (1.12.253)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.253->boto3->transformers>=2.0.0->fast-bert==1.4.4) (0.15.2)\n",
            "Building wheels for collected packages: fast-bert\n",
            "  Building wheel for fast-bert (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fast-bert: filename=fast_bert-1.4.4-cp36-none-any.whl size=59914 sha256=8794e22304883b11509bd1e2c12103834706812ac83e43152c8b15e85a64d07d\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-_i5zhlmk/wheels/53/11/ef/67d507c8f4f05b843a6dd2c4f6360acdee4c79679c4ca36bc6\n",
            "Successfully built fast-bert\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oiUTaoul0oZu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "ef30725b-c0d1-4327-93d2-047b75994070"
      },
      "source": [
        "!pip install python-box\n",
        "!pip install apex"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: python-box in /usr/local/lib/python3.6/dist-packages (3.4.5)\n",
            "Collecting apex\n",
            "  Downloading https://files.pythonhosted.org/packages/31/b6/923de12ffcc2686157d7f74b96396b87c854eaaeec9d441d120facc2a0e0/apex-0.9.10dev.tar.gz\n",
            "Collecting cryptacular\n",
            "  Downloading https://files.pythonhosted.org/packages/ec/d6/a82d191ec058314b2b7cbee5635150f754ba1c6ffc05387bc9a57efe48b8/cryptacular-1.5.5.tar.gz\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting zope.sqlalchemy\n",
            "  Downloading https://files.pythonhosted.org/packages/a1/43/b82d9b45a9c990ddadc554ec609442e2857f6347b6f819dc628b0748dbb5/zope.sqlalchemy-1.2-py2.py3-none-any.whl\n",
            "Collecting velruse>=1.0.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9c/d9/e18b5c98667c45f5dd1a256d72168ea5ff68f0025fc5b24be010f2696ca3/velruse-1.1.1.tar.gz (709kB)\n",
            "\u001b[K     |████████████████████████████████| 716kB 7.9MB/s \n",
            "\u001b[?25hCollecting pyramid>1.1.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4e/4f/6fe39af43fadc6d6c12f4cff9ed438f4fed20245614170959b38fe9f762d/pyramid-1.10.4-py2.py3-none-any.whl (325kB)\n",
            "\u001b[K     |████████████████████████████████| 327kB 42.5MB/s \n",
            "\u001b[?25hCollecting pyramid_mailer\n",
            "  Downloading https://files.pythonhosted.org/packages/ea/c3/0ce593179a8da8e1ab7fe178b0ae096a046246bd44a5787f72940d6dd5b2/pyramid_mailer-0.15.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from apex) (2.21.0)\n",
            "Collecting wtforms\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9f/c8/dac5dce9908df1d9d48ec0e26e2a250839fa36ea2c602cc4f85ccfeb5c65/WTForms-2.2.1-py2.py3-none-any.whl (166kB)\n",
            "\u001b[K     |████████████████████████████████| 174kB 43.2MB/s \n",
            "\u001b[?25hCollecting wtforms-recaptcha\n",
            "  Downloading https://files.pythonhosted.org/packages/5c/b0/42021ab061b768e3e5f430466219468c2afec99fe706e4340792d7a6fab4/wtforms_recaptcha-0.3.2-py2.py3-none-any.whl\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from cryptacular->apex) (41.4.0)\n",
            "Collecting pbkdf2\n",
            "  Downloading https://files.pythonhosted.org/packages/02/c0/6a2376ae81beb82eda645a091684c0b0becb86b972def7849ea9066e3d5e/pbkdf2-1.3.tar.gz\n",
            "Collecting transaction>=1.6.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/21/38/499bcc737411b00ea969fdc82e1ff56925a0ea634fe03173e18fbdd59015/transaction-2.4.0-py2.py3-none-any.whl (48kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 7.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: SQLAlchemy>=0.7 in /usr/local/lib/python3.6/dist-packages (from zope.sqlalchemy->apex) (1.3.10)\n",
            "Collecting zope.interface>=3.6.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/19/17/1d198a6aaa9aa4590862fe3d3a2ed7dd808050cab4eebe8a2f2f813c1376/zope.interface-4.6.0-cp36-cp36m-manylinux1_x86_64.whl (167kB)\n",
            "\u001b[K     |████████████████████████████████| 174kB 45.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests-oauthlib in /usr/local/lib/python3.6/dist-packages (from velruse>=1.0.3->apex) (1.2.0)\n",
            "Collecting anykeystore\n",
            "  Downloading https://files.pythonhosted.org/packages/aa/dc/c4399c0e6b835710763705220f9c37681683f950678db799a5c7eda9e154/anykeystore-0.2.tar.gz\n",
            "Collecting python3-openid\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bd/de/52c5699f52dcee3037db587196dcaf63ffedf5fbeba3183afe9b21a3a89f/python3_openid-3.1.0-py3-none-any.whl (130kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 40.6MB/s \n",
            "\u001b[?25hCollecting plaster\n",
            "  Downloading https://files.pythonhosted.org/packages/61/29/3ac8a5d03b2d9e6b876385066676472ba4acf93677acfc7360b035503d49/plaster-1.0-py2.py3-none-any.whl\n",
            "Collecting webob>=1.8.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/06/e1/4acd2b4327fceb4c6446bdbca515f807ab83188526fd654940c00bcf8cc3/WebOb-1.8.5-py2.py3-none-any.whl (113kB)\n",
            "\u001b[K     |████████████████████████████████| 122kB 49.7MB/s \n",
            "\u001b[?25hCollecting zope.deprecation>=3.5.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f9/26/b935bbf9d27e898b87d80e7873a0200cebf239253d0afe7a59f82fe90fff/zope.deprecation-4.4.0-py2.py3-none-any.whl\n",
            "Collecting hupper>=1.5\n",
            "  Downloading https://files.pythonhosted.org/packages/eb/24/3eeb567055e7c02ee3c2a6d0561c25dc3401262e1b5646e97fb5bc4b1b29/hupper-1.9-py2.py3-none-any.whl\n",
            "Collecting venusian>=1.0\n",
            "  Downloading https://files.pythonhosted.org/packages/21/80/8e2ac4aef69e927754277d29636b65eceb4767e9da90a2c3046c4176184a/venusian-1.2.0-py2.py3-none-any.whl\n",
            "Collecting translationstring>=0.4\n",
            "  Downloading https://files.pythonhosted.org/packages/26/e7/9dcf5bcd32b3ad16db542845ad129c06927821ded434ae88f458e6190626/translationstring-1.3-py2.py3-none-any.whl\n",
            "Collecting plaster-pastedeploy\n",
            "  Downloading https://files.pythonhosted.org/packages/11/c4/0470056ea324c7a420c22647be512dec1b5e32b1b6e77e27c61838d2811c/plaster_pastedeploy-0.7-py2.py3-none-any.whl\n",
            "Collecting repoze.sendmail>=4.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/25/98/c5c64dc045b7c45858c391d04673a0f2748acef8e0eea4f2989b22220f97/repoze.sendmail-4.4.1-py2.py3-none-any.whl (41kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 6.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->apex) (2.8)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->apex) (1.24.3)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->apex) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->apex) (2019.9.11)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib->velruse>=1.0.3->apex) (3.1.0)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.6/dist-packages (from python3-openid->velruse>=1.0.3->apex) (0.6.0)\n",
            "Collecting PasteDeploy>=2.0\n",
            "  Downloading https://files.pythonhosted.org/packages/67/0c/faa9971b2e5e048b3b30008d04c72e4d5f63b42f48937c169acce2c5e70a/PasteDeploy-2.0.1-py2.py3-none-any.whl\n",
            "Building wheels for collected packages: cryptacular\n",
            "  Building wheel for cryptacular (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for cryptacular: filename=cryptacular-1.5.5-cp36-abi3-manylinux2010_x86_64.whl size=47560 sha256=870ed712df2654d6413723aa5ee9fb153f223e65f8f19c3cec39b894efb2b31f\n",
            "  Stored in directory: /root/.cache/pip/wheels/3c/79/bc/1eec7120c3ff9b0a2c7ad94d1626abc3388688e2ed7a45878f\n",
            "Successfully built cryptacular\n",
            "Building wheels for collected packages: apex, velruse, pbkdf2, anykeystore\n",
            "  Building wheel for apex (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for apex: filename=apex-0.9.10.dev0-cp36-none-any.whl size=46467 sha256=781e0cf30b94eacf9529fd66f1ee946f1707957cbcc7e3f78b19d16b0486085b\n",
            "  Stored in directory: /root/.cache/pip/wheels/39/4c/4b/2990cf86a29c679ae4bd5f4de5723aa8a4af107721089c9a55\n",
            "  Building wheel for velruse (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for velruse: filename=velruse-1.1.1-cp36-none-any.whl size=50924 sha256=11f9d818c400b34d3c6d7a3d7770fbbf62c594b52814574c3e523610fd82ad1f\n",
            "  Stored in directory: /root/.cache/pip/wheels/ca/65/9e/b805aad8ec3a359591c497b257dabe911f305d285b5d8a13cc\n",
            "  Building wheel for pbkdf2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pbkdf2: filename=pbkdf2-1.3-cp36-none-any.whl size=5105 sha256=210b25c6fe12b56806ddf188c2eb7599a9a19b3139259f9dbe0b10cfea9fbbd8\n",
            "  Stored in directory: /root/.cache/pip/wheels/8e/62/b9/0bf3a68f2111e169253ec4d2bbdc303c46777b7fc99bbbf230\n",
            "  Building wheel for anykeystore (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for anykeystore: filename=anykeystore-0.2-cp36-none-any.whl size=17027 sha256=28b296e39d3058b2d5d4132158e1788a16da0db887ef9057d13ca1c7cb22dbd1\n",
            "  Stored in directory: /root/.cache/pip/wheels/42/a9/b2/f79f84cbee6613c9edce6d98b9e1410c1d41d38953bd94eed2\n",
            "Successfully built apex velruse pbkdf2 anykeystore\n",
            "Installing collected packages: pbkdf2, cryptacular, zope.interface, transaction, zope.sqlalchemy, plaster, webob, zope.deprecation, hupper, venusian, translationstring, PasteDeploy, plaster-pastedeploy, pyramid, anykeystore, python3-openid, velruse, repoze.sendmail, pyramid-mailer, wtforms, wtforms-recaptcha, apex\n",
            "Successfully installed PasteDeploy-2.0.1 anykeystore-0.2 apex-0.9.10.dev0 cryptacular-1.5.5 hupper-1.9 pbkdf2-1.3 plaster-1.0 plaster-pastedeploy-0.7 pyramid-1.10.4 pyramid-mailer-0.15.1 python3-openid-3.1.0 repoze.sendmail-4.4.1 transaction-2.4.0 translationstring-1.3 velruse-1.1.1 venusian-1.2.0 webob-1.8.5 wtforms-2.2.1 wtforms-recaptcha-0.3.2 zope.deprecation-4.4.0 zope.interface-4.6.0 zope.sqlalchemy-1.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yS2_UlH1xTSM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Language model Databunch\n",
        "from fast_bert.data_lm import BertLMDataBunch\n",
        "# Language model learner\n",
        "from fast_bert.learner_lm import BertLMLearner\n",
        "\n",
        "from pathlib import Path\n",
        "from box import Box"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UqYnkzhHuK3K",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 382
        },
        "outputId": "e0c55b87-ed80-4633-8338-a40689b36fee"
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "from pathlib import Path\n",
        "import torch\n",
        "\n",
        "from box import Box\n",
        "import pandas as pd\n",
        "import collections\n",
        "import os\n",
        "from tqdm import tqdm, trange\n",
        "import sys\n",
        "import random\n",
        "import numpy as np\n",
        "import apex\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import datetime\n",
        "\n",
        "from fast_bert.modeling import BertForMultiLabelSequenceClassification\n",
        "from fast_bert.data_cls import BertDataBunch, InputExample, InputFeatures, MultiLabelTextProcessor, convert_examples_to_features\n",
        "from fast_bert.learner_cls import BertLearner\n",
        "from fast_bert.metrics import accuracy_multilabel, accuracy_thresh, fbeta, roc_auc\n"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-28-d9b19d446d8c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mapex\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_selection\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/apex/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m from apex.exceptions import (ApexAuthSecret,\n\u001b[1;32m     17\u001b[0m                              ApexSessionSecret)\n\u001b[0;32m---> 18\u001b[0;31m from apex.interfaces import (ApexImplementation,\n\u001b[0m\u001b[1;32m     19\u001b[0m                              IApex)\n\u001b[1;32m     20\u001b[0m from apex.lib.libapex import (groupfinder,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/apex/interfaces.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mApexImplementation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \"\"\" Class so that we can tell if Apex is installed from other \n\u001b[1;32m     12\u001b[0m     \u001b[0mapplications\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/apex/interfaces.py\u001b[0m in \u001b[0;36mApexImplementation\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mapplications\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \"\"\"\n\u001b[0;32m---> 14\u001b[0;31m     \u001b[0mimplements\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mIApex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/zope/interface/declarations.py\u001b[0m in \u001b[0;36mimplements\u001b[0;34m(*interfaces)\u001b[0m\n\u001b[1;32m    481\u001b[0m     \u001b[0;31m# the coverage for this block there. :(\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    482\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mPYTHON3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 483\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_ADVICE_ERROR\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m'implementer'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    484\u001b[0m     \u001b[0m_implements\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"implements\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minterfaces\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclassImplements\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    485\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: Class advice impossible in Python3.  Use the @implementer class decorator instead."
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fyar1lDe1XC2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 347
        },
        "outputId": "f15f5f85-37d6-46a3-ce26-1e04a72dd89b"
      },
      "source": [
        "!git clone https://www.github.com/NVIDIA/apex\n",
        "!cd apex\n",
        "!pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\""
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'apex' already exists and is not an empty directory.\n",
            "/usr/local/lib/python3.6/dist-packages/pip/_internal/commands/install.py:283: UserWarning: Disabling all use of wheels due to the use of --build-options / --global-options / --install-options.\n",
            "  cmdoptions.check_install_build_global(options)\n",
            "Created temporary directory: /tmp/pip-ephem-wheel-cache-gym3__6v\n",
            "Created temporary directory: /tmp/pip-req-tracker-1xf2tysb\n",
            "Created requirements tracker '/tmp/pip-req-tracker-1xf2tysb'\n",
            "Created temporary directory: /tmp/pip-install-gt7s85sg\n",
            "Cleaning up...\n",
            "Removed build tracker '/tmp/pip-req-tracker-1xf2tysb'\n",
            "\u001b[31mERROR: You must give at least one requirement to install (see \"pip help install\")\u001b[0m\n",
            "Exception information:\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/cli/base_command.py\", line 153, in _main\n",
            "    status = self.run(options, args)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/commands/install.py\", line 362, in run\n",
            "    wheel_cache\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/cli/req_command.py\", line 275, in populate_requirement_set\n",
            "    '(see \"pip help %(name)s\")' % opts)\n",
            "pip._internal.exceptions.CommandError: You must give at least one requirement to install (see \"pip help install\")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oDYWMpR7uWFc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}